Smart Refrigerator Management System

Project group picture

# Fengkai Tang (ft243) & Zekai Li (zl874)
  # Thursday Lab, Group 8
  # ECE 5725 Fall 2023 Final Project
  # menu.py
  import pygame
  from pygame.locals import *
  import os
  import time
  import RPi.GPIO as GPIO
  import subprocess
  import math
  from threading import Thread
  import threading
  import board
  import busio
  import adafruit_am2320
  import cv2
  from picamera import PiCamera
  from picamera.array import PiRGBArray
  from pyzbar import pyzbar
  from datetime import datetime
  import requests
  
  i2c = busio.I2C(board.SCL,board.SDA)
  am = adafruit_am2320.AM2320(i2c,address=0x5c)
  
  def get_temperature():
    while True:
      try:
        t = am.temperature
        return t
      except Exception as e:
        print("Error reading from the sensor: ",e)
        #print()
      
  def get_humidity():
    while True:
      try:
        h = am.relative_humidity
        return h
      except Exception as e:
        print("Error reading from the sensor: ",e)
        #print()
  
  def display_menu(button,font):
    for my_text,text_pos in button.items():
      text_surface = font.render(my_text, True, WHITE)
      rect = text_surface.get_rect(center=text_pos)
      screen.blit(text_surface, rect)
  
  def display_menu_red(button,font):
    for my_text,text_pos in button.items():
      text_surface = font.render(my_text, True, (255,69,0))
      rect = text_surface.get_rect(center=text_pos)
      screen.blit(text_surface, rect)
  
  def read_barcodes(frame):
    barcodes = pyzbar.decode(frame)
    barcode_detected = False
    barcode_text = ''
    for barcode in barcodes:
      barcode_detected = True
      x, y, w, h = barcode.rect
          # 1. Draw rectangle around the barcode
      cv2.rectangle(frame, (x, y), (x+w, y+h), (0, 255, 0), 2)
  
          # 2. Extract the barcode data as a string
      barcode_text = barcode.data.decode('utf-8')
      barcode_type = barcode.type
  
          # 3. Print the barcode type and data to the console
      print(f"Barcode: {barcode_text} | Type: {barcode_type}")
  
    return frame, barcode_detected, barcode_text
    # return frame, barcode_detected
  
  # call function in put in application to read the barcode
  def real_time_reading(W,H):
    global flag
    camera = PiCamera()
    camera.resolution = (W, H) # put in function use W = 640, H = 480
    camera.framerate = 24
    rawCapture = PiRGBArray(camera, size=(W, H))
    camera.rotation=270
    time.sleep(0.1)
    for frame in camera.capture_continuous(rawCapture, format="bgr", use_video_port=True):
      image = frame.array
      image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
      image, successBarcode, barcode = read_barcodes(image)
      # image, successBarcode = read_barcodes(image)
      image = pygame.image.frombuffer(image, (W, H),'RGB')
      screen.fill(Black)
      screen.blit(image, (0, 0))
      pygame.display.flip()
      # cv2.imshow("Barcode Scanner", image)
      key = cv2.waitKey(1) & 0xFF
  
      # Clear the stream for the next frame
      rawCapture.truncate(0)
            
      # if stuck, press 23 button
      if not flag:
        # barcode_menu = {'Type in expire date':(160,30),'':()}
        camera.close()
        return None
        break
      if successBarcode:
        camera.close()
        name = read_productname(barcode)
        flag = False
        return name
        break
  
  def read_productname(barcode):
    url = "https://barcodes-lookup.p.rapidapi.com/"
    querystring = {"query":barcode}
    headers = {
      "X-RapidAPI-Key": "30d80db95emshcbfe496918fa110p150398jsnd0ddbb49171f",
      "X-RapidAPI-Host": "barcodes-lookup.p.rapidapi.com"
    }
    response = requests.get(url, headers=headers, params=querystring)
    if response.status_code == 200:
      data = response.json()
      title = data['product']['title']
      return title
    else:
      return None
  
  def checklocation():
    global doorclosed
    global captured_images
    status = 1
    center_x = 0
    center_y = 0
    if doorclosed:
      if status == 1:
        camera.capture(rawCapture, format="bgr")
        image = rawCapture.array
              
        if len(captured_images) >= 2:
          captured_images.pop(0)
                  
        captured_images.append(image)
        rawCapture.truncate(0)
        print("One image captured, processing...")
        # doorclosed  = False
        camera.close()
        if len(captured_images) == 2:
          print("Two images captured, processing...")
          image1, image2 = captured_images
          # Convert to grayscale
          gray1 = cv2.cvtColor(image1, cv2.COLOR_BGR2GRAY)
          gray2 = cv2.cvtColor(image2, cv2.COLOR_BGR2GRAY)
  
          # Apply Gaussian blur
          blur1 = cv2.GaussianBlur(gray1, (5, 5), 0)
          blur2 = cv2.GaussianBlur(gray2, (5, 5), 0)
  
          # Find the difference
          difference = cv2.absdiff(blur1, blur2)
  
          # Threshold the difference
          _, threshold = cv2.threshold(difference, 50, 255, cv2.THRESH_BINARY)
  
          # Define kernel for morphological operations
          kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (5, 5))
  
          # Apply opening (erosion followed by dilation)
          opened = cv2.morphologyEx(threshold, cv2.MORPH_OPEN, kernel)
  
          # Apply closing (dilation followed by erosion)
          closed = cv2.morphologyEx(opened, cv2.MORPH_CLOSE, kernel)
  
          # Find contours on the closed image
          contours, _ = cv2.findContours(closed, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
  
          # Draw contours on the original image
          largest_contour = None
          largest_area = 0
  
          for contour in contours:
            area = cv2.contourArea(contour)
            if area > largest_area:  # Only save the largest contour
              largest_area = area
              largest_contour = contour
  
          if largest_contour is not None and largest_area > 4000:
          # Filter out small changes
            (x, y, w, h) = cv2.boundingRect(largest_contour)
            cv2.rectangle(image1, (x, y), (x+w, y+h), (0, 255, 0), 2)
            cv2.rectangle(image2, (x, y), (x+w, y+h), (0, 255, 0), 2)
            center_x, center_y = x + w//2, y + h//2
            print(f"Center: {center_x},{center_y}")
    
    return center_x, center_y
      
  
  # for exit the threading and the main while loop at the same time
  exit_event = threading.Event()
  
  # initialize the x and y coordinate
  x = 0
  y = 0
  pos = 0,0
  
  # detect x and y coordinate, use thread
  def task():
    global x
    global y
    global pos
    pygame.init()
    while not exit_event.is_set():
      for event in pygame.event.get():
        if(event.type is MOUSEBUTTONDOWN):
          pos = pygame.mouse.get_pos()
          x,y = pos
          print(f"mouse down {x},{y}")
        elif(event.type is MOUSEBUTTONUP):
          pos = pygame.mouse.get_pos()
          x,y = 0,0
          pos = 0,0
      pygame.time.delay(100)
  
  GPIO.setmode(GPIO.BCM) 
  # close the project
  GPIO.setup(27, GPIO.IN, pull_up_down=GPIO.PUD_UP)
  # no use
  GPIO.setup(23, GPIO.IN, pull_up_down=GPIO.PUD_UP)
  # E-Switch interrupt
  GPIO.setup(26, GPIO.IN, pull_up_down=GPIO.PUD_UP)
  # shutdown the system
  GPIO.setup(22, GPIO.IN, pull_up_down=GPIO.PUD_UP)
  os.putenv('SDL_VIDEODRIVER','fbcon')
  os.putenv('SDL_FBDEV','/dev/fb0')
  os.putenv('SDL_MOUSEDRV','TSLIB')
  os.putenv('SDL_MOUSEDEV','/dev/input/touchscreen')
  
  def GPIO_27_callback(channel):
    global MainMenu
    global PutInMenu
    global TakeOutMenu
    global StoredItemsMenu
    MainMenu = False
    PutInMenu = False
    TakeOutMenu = False
    StoredItemsMenu = False
    exit_event.set() # exit the threading at the same time
  
  def GPIO_23_callback(channel):
    global flag
    flag = not flag
  
  # E-switch interrupt
  def capture_image(channel):
      global doorclosed 
      doorclosed = True
  
  def shutdown_callback(channel):
    # we need while 1 to paint pitft black
    # without while 1, the main loop will paint pitft again, which causes no black screen when shutdown
    while 1:
      screen.fill(Black)
      pygame.display.flip()
      time.sleep(2)
      os.system("sudo shutdown -h now")
  
  flag = True
  doorclosed = False
  GPIO.add_event_detect(27,GPIO.FALLING,callback=GPIO_27_callback,bouncetime=300)
  GPIO.add_event_detect(23,GPIO.FALLING,callback=GPIO_23_callback,bouncetime=300)
  GPIO.add_event_detect(26 ,GPIO.FALLING, callback=capture_image, bouncetime=500)
  GPIO.add_event_detect(22 ,GPIO.FALLING, callback=shutdown_callback, bouncetime=500)
  MainMenu = True
  
  # function to draw the number keyboard
  def draw_keyboard():
      # screen.fill(Black)
      for key in keys:
          pygame.draw.rect(screen, GREY, key['rect'])
          font = pygame.font.Font(None, 30)
          label = font.render(key['label'], True, WHITE)
          label_rect = label.get_rect(center=key['rect'].center)
          screen.blit(label, label_rect)
      pygame.display.update()
      # pygame.display.flip()
  
  # get the position
  def get_key(pos):
      for key in keys:
          if key['rect'].collidepoint(pos):
              return key['label']
      return None
  
  # number layout
  key_width = 320 // 3
  key_height = (240 - 70) // 4  # 70 pixels for the text area
  keys = [
      {'rect': pygame.Rect(0, 70, key_width, key_height), 'label': '1'},
      {'rect': pygame.Rect(key_width, 70, key_width, key_height), 'label': '2'},
      {'rect': pygame.Rect(key_width * 2, 70, key_width, key_height), 'label': '3'},
      {'rect': pygame.Rect(0, 70 + key_height, key_width, key_height), 'label': '4'},
      {'rect': pygame.Rect(key_width, 70 + key_height, key_width, key_height), 'label': '5'},
      {'rect': pygame.Rect(key_width * 2, 70 + key_height, key_width, key_height), 'label': '6'},
      {'rect': pygame.Rect(0, 70 + key_height * 2, key_width, key_height), 'label': '7'},
      {'rect': pygame.Rect(key_width, 70 + key_height * 2, key_width, key_height), 'label': '8'},
      {'rect': pygame.Rect(key_width * 2, 70 + key_height * 2, key_width, key_height), 'label': '9'},
      {'rect': pygame.Rect(key_width, 70 + key_height * 3, key_width, key_height), 'label': '0'},
  ]
  
  # database part 
  # initialize a list to store the objects
  inventory = []
  
  # define a function to add object into the refrigerator
  def add_item(name, expiry_date, location,days):
      # we can only store 3 items in the refrigerator, if >= 3, cannot add any object
      if len(inventory) >= 3:
          print("Inventory is full. Cannot add more items.")
          return
      
      # we only have three loactions A,B,C in the refrigerator
      # one location can only store one object at a time
      if location not in ['A', 'B', 'C']:
          print("Invalid location. Use 'A', 'B', or 'C'.")
          return
      
      # create a dictionary to store items information
      # we store the name, expiry date and the location
      item = {
          'name': name,
          'expiry_date': expiry_date,
          'location': location,
    'days' : days
      }
      # append the item to the inventory list
      inventory.append(item)
  
  # define a function to display the information of all the stored items
  def show_inventory():
      for item in inventory:
          print(item)
  
  # remove item from inventory list (item is removed from the refrigerator)
  def remove_item(location):
      global inventory
      inventory = [item for item in inventory if item['location'] != location]
  
  # add_item('Poland Spring', '2023-11-28', 'A','20')
  # add_item('Pepsi Diet Coke', '2023-12-05', 'B',"expire")
  # add_item('Eggs', '2024-01-15', 'C',"200")
  # show_inventory()
  # remove_item('A')
  # show_inventory()
  
  # for display on the pitft
  def draw_inventory(inventory):
    inventory_menu_font = pygame.font.Font(None, 22)
      # enumerate all the items in inventory
    for index, item in enumerate(inventory):
      inventory_menu = {item['name']:(80,index*30+60),item['expiry_date']:(200,index*30+60),item['location']:(280,index*30+60)}
      if item['days'] == "expire":
        remaining_menu = {item['name']:(80,index*30+150)}
        red_menu = {"expired already":(200,index*30+150)}
        display_menu(inventory_menu,inventory_menu_font)
        display_menu(remaining_menu,inventory_menu_font)
        display_menu_red(red_menu,inventory_menu_font)
      else:
        remaining_menu = {item['name']:(80,index*30+150),item['days']:(170,index*30+150),"d left":(200,index*30+150)}
      
        display_menu(inventory_menu,inventory_menu_font)
        display_menu(remaining_menu,inventory_menu_font)
    # pygame.display.flip()
  
  pygame.init()
  
  pygame.mouse.set_visible(False)
  size = width, height = 320,240
  WHITE = 255, 255, 255
  Black = 0, 0, 0
  GREY = 150, 150, 150
  screen = pygame.display.set_mode(size)
  
  main_menu_font = pygame.font.Font(None,30)
  main_menu ={'Put in':(80,120),'Take out':(220,120),'Stored Items':(160,200)} # T (60,50) , H (200,50)
  
  put_in_menu_font = pygame.font.Font(None,30)
  put_in_menu = {'Please open the door':(160,30),'to scan the barcode':(160,70),'Open the Camera':(160,160),'Back':(280,200)}
  
  take_out_menu_font = pygame.font.Font(None,30)
  take_out_menu = {'Please take out one item ':(160,30),'Back':(280,200),'and close the door':(160,70), 'Complete':(60, 200)}
  
  
  stored_items_menu_font = pygame.font.Font(None,25)
  stored_items_menu = {'Name':(80,30),'Expiry Date':(200,30),'Loc.':(280,30),'Back':(290,220)}
  
  put_in_second_menu = {'Please enter expire date as yyyy-mm-dd':(160,30)}
  put_in_second_menu_font = pygame.font.Font(None,20)
  
  put_in_close_door_menu = {'Successful scan':(160,30),'Wait for door close':(160,70)}
  put_in_close_door_menu_font = pygame.font.Font(None,25)
  
  finish_type_menu = {'Succuessful enter!':(160,30),'Exit in 5 seconds':(160,120)}
  finish_type_menu_font = pygame.font.Font(None,30)
  
  remove_menu = {'Succuessful removal!':(160,30),'Exit in 5 seconds':(160,120)}
  remove_menu_font = pygame.font.Font(None,30)
  
  temperature = get_temperature()
  humidity = get_humidity()
  temp_hum = {str(temperature):(120,50),'°C':(150,50),str(humidity):(260,50),'%':(290,50)}
  
  # screen.fill(Black)
  # display_menu(main_menu,main_menu_font)
  # display_menu(temp_hum,main_menu_font)	
  # pygame.display.flip()
  
  Touch = Thread(target = task)
  Touch.start()
  PutInMenu = True
  
  # make a photo here
  camera = PiCamera() # Set up Camera
  camera.rotation = 270
  camera.resolution = (640, 480)
  rawCapture = PiRGBArray(camera, size=(640, 480))
  time.sleep(1)
  captured_images = [] # Array to store images 
  camera.capture(rawCapture, format="bgr") # Capture image
  image = rawCapture.array # Covert the image to mat format for futher processing
  captured_images.append(image)
  rawCapture.truncate(0)
  camera.close()
  
  # load temperature symbol
  temp_symbol = pygame.image.load("/home/pi/final_project/temp.png")
  # load humudity symbol
  humidity_symbol = pygame.image.load("/home/pi/final_project/humidity.jpg")
  # resize temperature symbol to fit pitft
  temp_symbol = pygame.transform.scale(temp_symbol,(20,40)) # 1:2
  # resize humidity symbol to fit pitft
  humidity_symbol = pygame.transform.scale(humidity_symbol,(40,48)) # 5:6
  
  while MainMenu:
    time.sleep(0.5)
    temperature = get_temperature()
    humidity = get_humidity()
    temp_hum = {str(temperature):(120,50),'°C':(150,50),str(humidity):(260,50),'%':(290,50)}
    screen.fill(Black)
    # draw temperature symbol
    screen.blit(temp_symbol, (70, 20))
    # draw humidity symbol
    screen.blit(humidity_symbol,(200,20))
    # rect for put in button
    pygame.draw.rect(screen,(255,255,255),(40,100,80,40),2) # screen, color, rect position, border_width
    # rect for take out button
    pygame.draw.rect(screen,(255,255,255),(170,100,100,40),2) # screen, color, rect position, border_width
    # rect for stored item button
    pygame.draw.rect(screen,(255,255,255),(80,180,160,40),2) # screen, color, rect position, border_width
    display_menu(main_menu,main_menu_font)
    display_menu(temp_hum,main_menu_font)
      
    pygame.display.flip()
    #put_in_menu = {'Please scan the barcode':(160,80),'Open the Camera':(160,160),'Back':(280,200)}
    #take_out_menu = {'Please open the door':(160,80),'Back':(280,200)}
    PutInMenu = True
    flag = True
    doorclosed = False
    # Put in Menu
    if 40 < x < 120 and 60 < y < 180:
      while PutInMenu:
        screen.fill(Black)
        pygame.draw.rect(screen,(255,255,255),(70,140,180,40),2) # screen, color, rect position, border_width
        display_menu(put_in_menu,put_in_menu_font)
        pygame.display.flip()
        
        # Open camera button
        if 120 < x < 200 and 120 < y < 200:
          product_title = real_time_reading(640,480)
          # shorten the words in title for display
          words = product_title.split()
          # 1 space to split
          # num_spaces = 1
          shorten_title = ''.join(words[:2])
              
          # check the item location
          doorclosed = False
          if not doorclosed:
            camera = PiCamera() # Set up Camera
            camera.rotation = 270
            camera.resolution = (640, 480)
            rawCapture = PiRGBArray(camera, size=(640, 480))
            while not doorclosed:
              screen.fill(Black)
              display_menu(put_in_close_door_menu,put_in_close_door_menu_font)
              pygame.display.flip()
              loc_x, loc_y = checklocation()
              if loc_x <= 213:
                location = 'A'
                # camera.close()
              elif loc_x >= 427:
                location = 'C'
              else:
                location = 'B'
              
              
            
        
        DATE = []
        while not flag:
          expire_date = ''.join(map(str,DATE))
          screen.fill(Black)
          number_menu = {expire_date:(160,60),'Skip':(270,220),'Del':(60,220)}
          display_menu(put_in_second_menu,put_in_second_menu_font)
          display_menu(number_menu,put_in_menu_font)
          draw_keyboard()
          num = get_key(pos)
          time.sleep(0.2)
          if len(DATE) == 8:
            expire_date = '{}{}{}{}-{}{}-{}{}'.format(*DATE)
            number_menu = {expire_date:(160,60)}
            input_date = datetime.strptime(expire_date,"%Y-%m-%d")
            current_date = datetime.now()
            remaining_days = (input_date-current_date).days
            if remaining_days < 0:
              remaining_days = str(remaining_days)
              remaining_days = "expire"
            screen.fill(Black)
            display_menu(finish_type_menu,finish_type_menu_font)
            display_menu(number_menu,put_in_menu_font)
            pygame.display.flip()
            add_item(shorten_title,expire_date,location,str(remaining_days))
            time.sleep(5)
            flag = True
            PutInMenu = False
            
          
          if num:
            DATE.append(num)
          
          # Delete button
          if 30 < x < 90 and 180 < y < 240:
            DATE.pop(-1)
            
          # Skip button
          if 260 < x < 300 and 180 < y < 240:
            flag = True
            add_item(shorten_title,'',location,'')
            time.sleep(0.2)
            
        # Back button
        if 260 < x < 300 and 180 < y < 220:
          PutInMenu = False
    
    # Take out menu
    TakeOutMenu = True
    if 200 < x < 240 and 60 < y < 180:
      while TakeOutMenu:
        screen.fill(Black)
        pygame.draw.rect(screen,(255,255,255),(10,180,100,40),2) # screen, color, rect position, border_width
        display_menu(take_out_menu,take_out_menu_font)
        pygame.display.flip()
              
        # compare the difference to tell which object is moved
        # Complete button
        if 0 < x < 100 and 180 < y < 220:
          camera = PiCamera() # Set up Camera
          camera.rotation = 270
          camera.resolution = (640, 480)
          rawCapture = PiRGBArray(camera, size=(640, 480))
          loc_x, loc_y = checklocation()
          if loc_x <= 213:
            remove_item('A')
          elif loc_x >= 427:
            remove_item('C')
          else:
            remove_item('B')
          screen.fill(Black)
          display_menu(remove_menu,remove_menu_font)
          pygame.display.flip()
          camera.close()
          time.sleep(5)
          break				
          
         # Back button
        if 260 < x < 300 and 180 < y < 220:
          TakeOutMenu = False
    
    # Stored Items menu
    draw_item = True
    StoredItemsMenu = True
    if 120 < x < 200 and 140 < y < 260:
      while StoredItemsMenu:
        screen.fill(Black)
        display_menu(stored_items_menu,stored_items_menu_font)
        
        draw_inventory(inventory)
        
        pygame.display.flip()
        
        # Back button
        if 260 < x < 300 and 180 < y < 220:
          StoredItemsMenu = False
        
  
  
  Touch.join()